import random
import time

from fake_useragent import UserAgent
from lxml import etree
import requests

class getLianJia(object):
    def __init__(self):
        self.url = 'https://bj.lianjia.com/ershoufang/'

    # 随机获取伪代理
    def getHeader(self):
        # 实例化ua对象
        ua = UserAgent()
        headers = {'User-Agent': ua.random}
        return headers

    def getHtml(self, url):
        res = requests.get(url=url, headers=self.getHeader(), timeout=3)
        myHtml = res.text
        print("获取到html请求的文本内容： " + myHtml)
        return myHtml

    def parseHtml(self, url):
        textHtml = self.getHtml(url)
        if textHtml:
            p = etree.HTML(textHtml)
            # 定义正则表达式并且获取节点对象
            hList = p.xpath('//ul[@class="sellListContent"/]li[@class="clear LOGVIEWDATA LOGCLICKDATA"]')
            print(hList)
            # 获取每个节点的信息
            for h in hList:
                item = {}
                # 名称
                nameList = h.xpath('.//a[@data-el="region"]/text()')
                item['name'] = nameList[0] if nameList else None
                # 房屋属性
                infoList = h.xpath('.//div[@class="houseInfo"]/text()')
                if infoList:
                    L = infoList[0].split('|')
                    # pinrt(L)
                    if (len(L)) >= 5:
                        item['model'] = L[0].strip()
                        item['area'] = L[1].strip()
                        item['direction'] = L[2].strip()
                # 区域
                addressList = h.xpath('.//div[@class="positonInfo"]/text()')
                item['address'] = addressList[0] if addressList else None

                print(item)

    def run(self):
        try:
            for i in range(1, 10):
                self.parseHtml(self.url)
                time.sleep(random.randint(1, 2))
        except Exception as e:
            print("发生异常：", e)


if __name__ == '__main__':
    lianjia = getLianJia()
    lianjia.run()